COMMENT ⊗   VALID 00003 PAGES
C REC  PAGE   DESCRIPTION
C00001 00001
C00002 00002	% Here is TEX material that gets inserted after \input webhdr
C00005 00003	@* Tokens.
C00013 ENDMK
C⊗;
% Here is TEX material that gets inserted after \input webhdr
\def\hang{\hangindent 3em\ \unskip\!}
\chcode'27=13 \def↔{\ifmmode{\mathrel{\char'444}}\else{\penalty999\ } }
	% make ↔ tie two words together except in math mode
\def\TEX{\hbox{T\hskip-.1667em\lower.424ex\hbox{E}\hskip-.125em X}}
\font b=cmr9 \def\mc{\:b} % medium caps for names like PASCAL
\def\PASCAL{{\mc PAS}\-{\mc CAL}}
\def\pb{$\.|\ldotsm\.|$} % pascal brackets (|...|)
\def\v{\.{\char'174}} % vertical (|) in typewriter font

\font D=cmtt at 15truept % font used in the title line below (only)
\font E=cmr7 at 14truept % font used in the title line below (only)
\topspace 0pt
\vfill
\ctrline{\:E The {\:D TANGLE} processor}
\vskip 15pt
\ctrline{(preliminary version, November 1981)}
\vfill
\ctrline{\baselineskip9pt\hbox par 5in{\:bThis copy of the program is
	being distributed to interested ``guinea pigs'' before it has
	been thoroughly tested, in order to get extra help in the testing
	process. It is expected that changes will be made frequently
	while \TEX82 is being developed in the next months. Several
	comments in this documentation imply that the program has
	already been applied successfully to \TEX; such remarks are
	merely 	wishful thinking at this point, but they will be true
	when the final version is completed.}}
\mark{1}\eject
@* Tokens.
Replacement texts, which represent \PASCAL\ code in a compressed format,
appear in |tok_mem| as mentioned above. The codes in
these texts are called `tokens'; some tokens occupy two consecutive
eight-bit byte positions, and the others take just one byte.

If $p>0$ points to a replacement text, |tok_start[p]| is the |tok_mem| position
of the first eight-bit code of that text. If |text_link[p]=0|,
this is the replacement text for a macro, otherwise it is the replacement
text for a module. In the latter case |text_link[p]| is either equal to
|module_flag|, which means that there is no further text for this module, or
|text_link[p]| points to a
continuation of this replacement text; such links are created when
several modules have \PASCAL\ texts with the same name, and they also
tie together all the \PASCAL\ texts of unnamed modules.
The replacement text pointer for the first unnamed module
appears in |text_link[0]|, and the most recent such pointer is |last_unnamed|.

@d module_flag==max_texts {final |link| in module replacement texts}

@<Glob...@>=
last_unnamed:text_pointer; {most recent replacement text of unnamed module}

@ @<Set init...@>= last_unnamed←0; text_link[0]←0;

@ If the first byte of a token is less than @'200, the token occupies a
single byte. Otherwise we make a sixteen-bit token by combining two consecutive
bytes $a$ and $b$. If $@'200≤a<@'250$, then $(a-@'200)\times2↑8+b$ points
to an identifier; if $@'250≤a<@'320$, then
$(a-@'250)\times2↑8+b$ points to a module name; otherwise, i.e., if
$@'320≤a<@'400$, then $(a-@'320)\times2↑8+b$ is the number of the module
in which the current replacement text appears.

Codes less than @'200 are 7-bit ascii codes that represent themselves.
In particular, a single-character identifier like `|x|' will be a one-byte
token, while all longer identifiers will occupy two bytes.

Some of the 7-bit ascii codes will not be present, however, so we can
use them for special purposes. The following symbolic names are used:

\yskip\hang |begin_comment| denotes \.{@@\{}, which will become either
\.{\{} or \.{[}.

\yskip\hang |end_comment| denotes \.{@@\}}, which will become either
\.{\}} or \.{]}.

\yskip\hang |octal| denotes the \.{@@\'} that precedes an octal constant.

\yskip\hang |octal| denotes the \.{@@@\'} that precedes an octal constant.

\yskip\hang |octal| denotes the \.{@@@'} that precedes an octal constant.

\yskip\hang |octal| denotes the |@'| that precedes an octal constant.

\yskip\hang |param| denotes insertion of a parameter. This occurs only in
the replacement texts of parametric macros, outside of single-quoted strings
in those texts.

\yskip\hang |join| denotes the concatenation of adjacent items with no
space or line breaks allowed between them (the |@@&| operation of \.{WEB}).

\yskip\hang |double_dot| denotes `\.{..}' in \PASCAL.

@d begin_comment=@'11 {ascii tab mark will not appear}
@d end_comment=@'12 {ascii line feed will not appear}
@d octal=@'14 {ascii form feed will not appear}
@d param=@'15 {ascii carriage return will not appear}
@d double_dot=@'40 {ascii space will not appear except in strings}
@d join=@'177 {ascii delete will not appear}

@ The following procedure is used to enter a two-byte value into
|tok_mem| when a replacement text is being generated.

@p procedure store_two_bytes(x:sixteen_bits); {stores high byte, then low byte}
begin if tok_ptr+2>max_toks then overflow('token');
tok_mem[tok_ptr]←x div@'400; {this could be done by a shift command}
tok_mem[tok_ptr+1]←x mod@'400; {this could be done by a logical and}
tok_ptr←tok_ptr+2;
end;

@ When \.{TANGLE} is being operated in debug mode, it has a procedure to display
a replacement text in symbolic form. This procedure has not been spruced up to
generate a real great format, but at least the results are not as bad as
a memory dump.

@p DEBUG procedure print_repl(p:text_pointer);
var k:0..max_toks; {index into |tok_mem|}
a: sixteen_bits; {current byte(s)}
begin if p≥text_ptr then print('BAD')
else	begin k←tok_start[p];
	while k<tok_start[p+1] do
		begin a←tok_mem[k];
		if a≥@'200 then @<Display two-byte token starting with $a$@>
		else @<Display one-byte token $a$@>;
		incr(k);
		end;
	end;
end;
GUBED

@ @<Display two-byte...@>=
begin incr(k);
if a<@'250 then {identifier or string}
	begin a←(a-@'200)*@'400+tok_mem[k]; print_id(a);
	if byte_mem[byte_start[a]]="""" then print('"')
	else print(' ');
	end
else if a<@'320 then {module name}
	begin print('@@<'); print_id((a-@'250)*@'400+tok_mem[k]);
	print('@@>');
	end
else	begin a←(a-@'320)*@'400+tok_mem[k]; {module number}
	print('@@{',a:0,'@@',chr("}")); {can't use right brace in comments}
	end;
end

@ @<Display one-byte...@>=
case a of
begin_comment: print('@@{');
end_comment: print('@@',chr("}")); {can't use right brace in comments}
octal: print('@@''');
param: print('#');
"@@": print('@@@@');
othercases print(chr(a))
endcases